ERDDAP Extraction

metadata: erddap_sss.yml

Published

2025-05-20 00:01 UTC

Code

# dir_extractr = "/share/github/marinebon/extractr"
# dir_extractr = "~/Github/marinebon/extractr" # DEBUG
# devtools::load_all(dir_extractr)
# devtools::install_local(dir_extractr, force = T)
# devtools::install_github("marinebon/extractr", force = T)
librarian::shelf(
  dplyr, DT, fs, glue, here, logger, lubridate, mapview, purrr, RColorBrewer, readr, 
  sf, stringr, terra, tibble, yaml,
  marinebon/extractr,
  quiet = T)
options(readr.show_col_types = F)
mapviewOptions(
  basemaps       = "Esri.OceanBasemap",
  vector.palette = colorRampPalette(brewer.pal(n=5, name="Spectral")))

if (!exists("params"))
  params <- list(yml = here("meta/erddap_sst.yml")) # DEBUG

1 Polygons

Code

sanctuaries_rds <- here("../climate-dashboard/data/sanctuaries.rds")

ply_sanctuaries <- readRDS(sanctuaries_rds)

# show map
mapView(ply_sanctuaries, zcol = "nms")

Code

ply_sanctuaries |>
  st_drop_geometry() |>
  datatable()

2 Dataset

2.1 Metadata

Contents of erddap_sss.yml:

Code

cat(
  "```yaml",
  readLines(params$yml),
  "```",
  sep = "\n")

erddap_url: https://coastwatch.noaa.gov/erddap/griddap/noaacwSMOSsss3day.html
erddap_variable: sss

2.2 Information

Code

meta <- read_yaml(params$yml) 
proc_prod <- path_ext_remove(basename(params$yml))

dir_out <- here(glue("data/{proc_prod}"))
dir_create(dir_out)

v     <- meta$erddap_variable
(ed   <- ed_info(meta$erddap_url))

<ERDDAP info> noaacwSMOSsss3day 
 Base URL: https://coastwatch.noaa.gov/erddap 
 Dataset Type: griddap 
 Dimensions (range):  
     time: (2010-06-03T12:00:00Z, 2025-05-10T12:00:00Z) 
     altitude: (0.0, 0.0) 
     latitude: (-89.875, 89.875) 
     longitude: (-179.875, 179.875) 
 Variables:  
     sss: 
         Units: PSU 
     sss_dif: 
         Units: PSU

Code

times <- ed_dim(ed, "time")

3 Polygon years

Code

d_nms_yr_todo <- ply_sanctuaries |> 
  st_drop_geometry() |> 
  arrange(nms) |> 
  select(nms) |> 
  cross_join(
    tibble(
      year = year(min(times)):year(max(times)))) |> 
  mutate(
    d = map2(nms, year, \(nms, year) { # nms = "CBNMS"; year = 2025
      
      times_yr <- times[year(times) == year]
      # csv <- glue("{dir_out}/{nms}/{year}.csv")
      # 
      # # time[1] if missing csv
      # if (!file.exists(csv))
      #   return(list(
      #     n_times  = length(times_yr),
      #     time_min = times_yr[1]))
      # 
      # times_csv <- read_csv(csv) |> 
      #   pull(time)
      # # NA if all times done
      # if (all(times_yr %in% times_csv))
      #   return(list(
      #     n_times  = 0,
      #     time_min = NA)) 
      # 
      # # earliest time missing otherwise
      # i <- !times_yr %in% times_csv
      # list(
      #     n_times  = sum(i),
      #     time_min = min(times_yr[i])) 
      tif <- glue("{dir_out}/{nms}/{year}.tif")
      
      # if missing tif,return all times
      if (!file.exists(tif))
        return(list(
          time_min = min(times_yr),
          time_max = max(times_yr),
          n_times  = length(times_yr)))
      
      # get times from tif
      r <- rast(tif)
      times_tif <- time(r)
      
      # if all times done, return NAs
      if (all(times_yr %in% times_tif))
        return(list(
          n_times  = 0,
          time_min = NA,
          time_max= NA)) 
      
      # otherwise, return time range missing
      i <- !times_yr %in% times_tif
      list(
          n_times  = sum(i),
          time_min = min(times_yr[i]),
          time_max = max(times_yr[i])) 
      }),
    time_min = map_vec(d, pluck, "time_min"),
    time_max = map_vec(d, pluck, "time_max"),
    n_times  = map_int(d, pluck, "n_times") ) |> 
  select(-d) |> 
  filter(n_times > 0) |> 
  rowid_to_column("i") |>
  relocate(i)

d_nms_yr_todo |> 
  group_by(nms) %>%
  {if (nrow(.) > 0)
    summarize(
      .,
      time_min = min(time_min, na.rm = T),
      time_max = min(time_max, na.rm = T),
      n_times  = sum(n_times)) else .} |>
  datatable(
    caption  = "Sanctuaries missing available ERDDAP times.",
    rownames = F,
    options  = list(
      pageLength = 5,
      lengthMenu = c(5, 50, nrow(d_nms_yr_todo))))

4 Extract dataset per polygon year

Using extractr::ed_extract(), .

Code

# rerddap::cache_delete_all()

fxn <- function(i, nms, time_min, time_max, ...){ #  nms = "GRNMS"; year = 2010 ; i = 97
  
  err <- tryCatch({
    
    log_info("{sprintf('%03d', i)}: {nms}, {time_min} to {time_max}")
    
    yr <- year(time_min)

    ply <- ply_sanctuaries |>
      filter(nms == !!nms)
    # TODO: consider expanding by 10% and rounding 2 digits
    # bb <- st_bbox(ply) |> stars:::bb_shrink(-0.1) |> round(2)
    
    # rast(glue("{dir_out}/{nms}/{yr}.tif")) |> names()
    # browser()
    # devtools::reload(dir_extractr) # DEBUG
    
    extractr::ed_extract( # DEBUG
    # ed_extract(
      ed,
      var       = v,
      sf_zones  = ply,
      mask_tif  = T,
      rast_tif  = glue("{dir_out}/{nms}/{yr}.tif"),
      zonal_fun = "mean",
      zonal_csv = glue("{dir_out}/{nms}/{yr}.csv"),
      dir_nc    = glue("{dir_out}/{nms}/{yr}_nc"),
      keep_nc   = F,
      n_max_vals_per_req = 1e+05,
      time_min  = time_min,
      time_max  = time_max,
      verbose   = T)
    
    return(NA)
  }, error = function(e) {
    
    log_error(conditionMessage(e))
    return(conditionMessage(e))
  })
  
  err
} 

res <- d_nms_yr_todo |> 
  # slice(1) |> # DEBUG
  mutate(
    error = pmap_chr(list(i, nms, time_min, time_max), fxn))

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2010-06-03 to 2010-12-24) ~ 00:00:47 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2011-01-13 to 2011-12-30) ~ 00:00:49 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2012-01-01 to 2012-12-29) ~ 00:00:51 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2013-01-01 to 2013-12-30) ~ 00:00:53 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2014-01-01 to 2014-12-30) ~ 00:00:56 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2015-01-01 to 2015-12-30) ~ 00:00:57 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2016-01-01 to 2016-12-29) ~ 00:00:59 UTC

Error in open.connection(structure(4L, class = c("curl", "connection"), conn_id = <pointer: 0x56129221f850>),  : 
  cannot open the connection

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2018-02-27 to 2018-12-30) ~ 00:01:04 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2019-01-01 to 2019-12-30) ~ 00:01:06 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2020-01-01 to 2020-12-29) ~ 00:01:08 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2021-01-01 to 2021-12-30) ~ 00:01:09 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2022-01-01 to 2022-12-30) ~ 00:01:10 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2023-01-01 to 2023-12-30) ~ 00:01:18 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2024-01-01 to 2024-12-29) ~ 00:01:21 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2025-01-01 to 2025-05-10) ~ 00:01:23 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2010-06-03 to 2010-12-24) ~ 00:01:24 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2011-01-13 to 2011-12-30) ~ 00:01:26 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2012-01-01 to 2012-12-29) ~ 00:01:28 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2013-01-01 to 2013-12-30) ~ 00:01:36 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2014-01-01 to 2014-12-30) ~ 00:01:37 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2015-01-01 to 2015-12-30) ~ 00:01:39 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2016-01-01 to 2016-12-29) ~ 00:01:40 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2017-01-01 to 2017-12-30) ~ 00:01:42 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2018-02-27 to 2018-12-30) ~ 00:01:44 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2019-01-01 to 2019-12-30) ~ 00:01:46 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2020-01-01 to 2020-12-29) ~ 00:01:47 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2021-01-01 to 2021-12-30) ~ 00:01:49 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2022-01-01 to 2022-12-30) ~ 00:01:51 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2023-01-01 to 2023-12-30) ~ 00:01:53 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2024-01-01 to 2024-12-29) ~ 00:01:54 UTC

Downloading 1 requests, up to 1e+05 time slices each

Fetching request 1 of 1 (2025-01-01 to 2025-05-10) ~ 00:01:56 UTC

Warning: There were 154 warnings in `mutate()`.
The first warning was:
ℹ In argument: `error = pmap_chr(list(i, nms, time_min, time_max), fxn)`.
Caused by warning in `new_CppObject_xp()`:
! GDAL Message 1: 1-pixel width/height files not supported, xdim: 1 ydim: 1
ℹ Run `dplyr::last_dplyr_warnings()` to see the 153 remaining warnings.

4.1 Successes

Code

res |> 
  mutate(
    success = is.na(error)) |> 
  group_by(success) |> 
  summarize(
    n = n()) |> 
  datatable()

4.2 Errors (if any)

Code

# rast(glue("{dir_out}/{nms}/{yr}.tif")) |> names()

res |> 
  filter(!is.na(error)) |> 
  group_by(nms) |> 
  summarize(
    n_years = n(),
    errors  = unique(error) |> paste(collapse = "\n\n----\n\n")) |> 
  datatable()

R package versions

Code

devtools::session_info()

─ Session info ───────────────────────────────────────────────────────────────
 setting  value
 version  R version 4.4.2 (2024-10-31)
 os       Ubuntu 24.04.1 LTS
 system   x86_64, linux-gnu
 ui       X11
 language (EN)
 collate  en_US.UTF-8
 ctype    en_US.UTF-8
 tz       Etc/UTC
 date     2025-05-20
 pandoc   3.5 @ /usr/bin/ (via rmarkdown)

─ Packages ───────────────────────────────────────────────────────────────────
 package           * version date (UTC) lib source
 base64enc           0.1-3   2015-07-28 [1] RSPM (R 4.4.0)
 bit                 4.6.0   2025-03-06 [1] RSPM (R 4.4.0)
 bit64               4.6.0-1 2025-01-16 [1] RSPM (R 4.4.0)
 brew                1.0-10  2023-12-16 [1] RSPM (R 4.4.0)
 bslib               0.9.0   2025-01-30 [1] RSPM (R 4.4.0)
 cachem              1.1.0   2024-05-16 [1] RSPM (R 4.4.0)
 class               7.3-22  2023-05-03 [2] CRAN (R 4.4.2)
 classInt            0.4-11  2025-01-08 [1] RSPM (R 4.4.0)
 cli                 3.6.5   2025-04-23 [1] RSPM (R 4.4.0)
 codetools           0.2-20  2024-03-31 [2] CRAN (R 4.4.2)
 colorspace          2.1-1   2024-07-26 [1] RSPM (R 4.4.0)
 crayon              1.5.3   2024-06-20 [1] RSPM (R 4.4.0)
 crosstalk           1.2.1   2023-11-23 [1] RSPM (R 4.4.0)
 crul                1.5.0   2024-07-19 [1] RSPM (R 4.4.0)
 curl                6.2.2   2025-03-24 [1] RSPM (R 4.4.0)
 data.table          1.17.2  2025-05-12 [1] RSPM (R 4.4.0)
 DBI                 1.2.3   2024-06-02 [1] RSPM (R 4.4.0)
 deldir              2.0-4   2024-02-28 [1] RSPM (R 4.4.0)
 devtools            2.4.5   2022-10-11 [1] RSPM (R 4.4.0)
 dichromat           2.0-0.1 2022-05-02 [1] RSPM (R 4.4.0)
 digest              0.6.37  2024-08-19 [1] RSPM (R 4.4.0)
 dplyr             * 1.1.4   2023-11-17 [1] RSPM (R 4.4.0)
 DT                * 0.33    2024-04-04 [1] RSPM (R 4.4.0)
 dygraphs            1.1.1.6 2018-07-11 [1] RSPM (R 4.4.0)
 e1071               1.7-16  2024-09-16 [1] RSPM (R 4.4.0)
 ellipsis            0.3.2   2021-04-29 [1] RSPM (R 4.4.0)
 evaluate            1.0.3   2025-01-10 [1] RSPM (R 4.4.0)
 extractr          * 0.1.7   2025-05-19 [1] local (/share/github/marinebon/extractr)
 farver              2.1.2   2024-05-13 [1] RSPM (R 4.4.0)
 fastmap             1.2.0   2024-05-15 [1] RSPM (R 4.4.0)
 fs                * 1.6.6   2025-04-12 [1] RSPM (R 4.4.0)
 generics            0.1.4   2025-05-09 [1] RSPM (R 4.4.0)
 glue              * 1.8.0   2024-09-30 [1] RSPM (R 4.4.0)
 here              * 1.0.1   2020-12-13 [1] RSPM (R 4.4.0)
 hms                 1.1.3   2023-03-21 [1] RSPM (R 4.4.0)
 hoardr              0.5.5   2025-01-18 [1] RSPM (R 4.4.0)
 htmltools           0.5.8.1 2024-04-04 [1] RSPM (R 4.4.0)
 htmlwidgets         1.6.4   2023-12-06 [1] RSPM (R 4.4.0)
 httpcode            0.3.0   2020-04-10 [1] RSPM (R 4.4.0)
 httpuv              1.6.16  2025-04-16 [1] RSPM (R 4.4.0)
 jquerylib           0.1.4   2021-04-26 [1] RSPM (R 4.4.0)
 jsonlite            2.0.0   2025-03-27 [1] RSPM (R 4.4.0)
 KernSmooth          2.23-24 2024-05-17 [2] CRAN (R 4.4.2)
 knitr               1.50    2025-03-16 [1] RSPM (R 4.4.0)
 later               1.4.2   2025-04-08 [1] RSPM (R 4.4.0)
 lattice             0.22-6  2024-03-20 [2] CRAN (R 4.4.2)
 leafem              0.2.4   2025-05-01 [1] RSPM (R 4.4.0)
 leaflet             2.2.2   2024-03-26 [1] RSPM (R 4.4.0)
 leaflet.providers   2.0.0   2023-10-17 [1] RSPM (R 4.4.0)
 leafpop             0.1.0   2021-05-22 [1] RSPM (R 4.4.0)
 librarian           1.8.1   2021-07-12 [1] RSPM (R 4.4.0)
 lifecycle           1.0.4   2023-11-07 [1] RSPM (R 4.4.0)
 logger            * 0.4.0   2024-10-22 [1] RSPM (R 4.4.0)
 lubridate         * 1.9.4   2024-12-08 [1] RSPM (R 4.4.0)
 magrittr            2.0.3   2022-03-30 [1] RSPM (R 4.4.0)
 mapview           * 2.11.2  2023-10-13 [1] RSPM (R 4.4.0)
 Matrix              1.7-1   2024-10-18 [2] CRAN (R 4.4.2)
 memoise             2.0.1   2021-11-26 [1] RSPM (R 4.4.0)
 mime                0.13    2025-03-17 [1] RSPM (R 4.4.0)
 miniUI              0.1.1.1 2018-05-18 [1] RSPM (R 4.4.0)
 ncdf4               1.24    2025-03-25 [1] RSPM (R 4.4.0)
 pillar              1.10.2  2025-04-05 [1] RSPM (R 4.4.0)
 pkgbuild            1.4.5   2024-10-28 [1] RSPM (R 4.4.0)
 pkgconfig           2.0.3   2019-09-22 [1] RSPM (R 4.4.0)
 pkgload             1.4.0   2024-06-28 [1] RSPM (R 4.4.0)
 png                 0.1-8   2022-11-29 [1] RSPM (R 4.4.0)
 polyclip            1.10-7  2024-07-23 [1] RSPM (R 4.4.0)
 profvis             0.4.0   2024-09-20 [1] RSPM (R 4.4.0)
 promises            1.3.2   2024-11-28 [1] RSPM (R 4.4.0)
 proxy               0.4-27  2022-06-09 [1] RSPM (R 4.4.0)
 purrr             * 1.0.4   2025-02-05 [1] RSPM (R 4.4.0)
 R.methodsS3         1.8.2   2022-06-13 [1] RSPM (R 4.4.0)
 R.oo                1.27.1  2025-05-02 [1] RSPM (R 4.4.0)
 R.utils             2.13.0  2025-02-24 [1] RSPM (R 4.4.0)
 R6                  2.6.1   2025-02-15 [1] RSPM (R 4.4.0)
 rappdirs            0.3.3   2021-01-31 [1] RSPM (R 4.4.0)
 raster              3.6-32  2025-03-28 [1] RSPM (R 4.4.0)
 RColorBrewer      * 1.1-3   2022-04-03 [1] RSPM (R 4.4.0)
 Rcpp                1.0.14  2025-01-12 [1] RSPM (R 4.4.0)
 readr             * 2.1.5   2024-01-10 [1] RSPM (R 4.4.0)
 remotes             2.5.0   2024-03-17 [1] RSPM (R 4.4.0)
 rerddap             1.2.1   2025-03-19 [1] RSPM (R 4.4.0)
 rlang               1.1.6   2025-04-11 [1] RSPM (R 4.4.0)
 rmarkdown           2.29    2024-11-04 [1] RSPM (R 4.4.0)
 rprojroot           2.0.4   2023-11-05 [1] RSPM (R 4.4.0)
 sass                0.4.10  2025-04-11 [1] RSPM (R 4.4.0)
 satellite           1.0.5   2024-02-10 [1] RSPM (R 4.4.0)
 scales              1.4.0   2025-04-24 [1] RSPM (R 4.4.0)
 sessioninfo         1.2.2   2021-12-06 [1] RSPM (R 4.4.0)
 sf                * 1.0-21  2025-05-15 [1] RSPM (R 4.4.2)
 shiny               1.9.1   2024-08-01 [1] RSPM (R 4.4.0)
 sp                  2.2-0   2025-02-01 [1] RSPM (R 4.4.0)
 spatstat.data       3.1-6   2025-03-17 [1] RSPM (R 4.4.0)
 spatstat.geom       3.3-6   2025-03-18 [1] RSPM (R 4.4.0)
 spatstat.univar     3.1-3   2025-05-08 [1] RSPM (R 4.4.0)
 spatstat.utils      3.1-4   2025-05-15 [1] RSPM (R 4.4.0)
 stringi             1.8.7   2025-03-27 [1] RSPM (R 4.4.0)
 stringr           * 1.5.1   2023-11-14 [1] RSPM (R 4.4.0)
 svglite             2.2.1   2025-05-12 [1] RSPM (R 4.4.0)
 systemfonts         1.2.3   2025-04-30 [1] RSPM (R 4.4.0)
 tabularaster        0.7.2   2023-11-01 [1] RSPM (R 4.4.0)
 terra             * 1.8-50  2025-05-09 [1] RSPM (R 4.4.0)
 textshaping         1.0.1   2025-05-01 [1] RSPM (R 4.4.0)
 tibble            * 3.2.1   2023-03-20 [1] RSPM (R 4.4.0)
 tidyr               1.3.1   2024-01-24 [1] RSPM (R 4.4.0)
 tidyselect          1.2.1   2024-03-11 [1] RSPM (R 4.4.0)
 timechange          0.3.0   2024-01-18 [1] RSPM (R 4.4.0)
 triebeard           0.4.1   2023-03-04 [1] RSPM (R 4.4.0)
 tzdb                0.5.0   2025-03-15 [1] RSPM (R 4.4.0)
 units               0.8-7   2025-03-11 [1] RSPM (R 4.4.0)
 urlchecker          1.0.1   2021-11-30 [1] RSPM (R 4.4.0)
 urltools            1.7.3   2019-04-14 [1] RSPM (R 4.4.0)
 usethis             3.1.0   2024-11-26 [1] RSPM (R 4.4.0)
 uuid                1.2-1   2024-07-29 [1] RSPM (R 4.4.0)
 vctrs               0.6.5   2023-12-01 [1] RSPM (R 4.4.0)
 vroom               1.6.5   2023-12-05 [1] RSPM (R 4.4.0)
 withr               3.0.2   2024-10-28 [1] RSPM (R 4.4.0)
 xfun                0.52    2025-04-02 [1] RSPM (R 4.4.0)
 xml2                1.3.8   2025-03-14 [1] RSPM (R 4.4.0)
 xtable              1.8-4   2019-04-21 [1] RSPM (R 4.4.0)
 xts                 0.14.1  2024-10-15 [1] RSPM (R 4.4.0)
 yaml              * 2.3.10  2024-07-26 [1] RSPM (R 4.4.0)
 zoo                 1.8-14  2025-04-10 [1] RSPM (R 4.4.0)

 [1] /usr/local/lib/R/site-library
 [2] /usr/local/lib/R/library

──────────────────────────────────────────────────────────────────────────────